{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Content\n",
    "## 1. Prepare videos\n",
    "## 2. Generate raw data list from DATA_PATH\n",
    "## 3. Confirm that videos are correct"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Prepare videos\n",
    "Please refer to the [official website](https://deepmind.com/research/open-source/kinetics) and/or the official script to prepare the videos.\n",
    "Note that the folder structure should look like this:\n",
    "```\n",
    "eva-video\n",
    "├── ...\n",
    "├── data\n",
    "│   ├── k400/600/700  -> PATH_TO_Kinetics-400/600/700\n",
    "│   │   ├── train\n",
    "│   │   │   ├── ${CLASS_NAME}/${VIDEO_ID}\n",
    "│   │   ├── val\n",
    "│   │   │   ├── ${CLASS_NAME}/${VIDEO_ID}\n",
    "│   ├── k400/600/700/722_train.txt\n",
    "│   ├── k400/600/700/722_val.txt\n",
    "│   ├── k722_to_k400/600/700_mapping.npy\n",
    "├── ...\n",
    "```"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Generate raw data list from DATA_PATH\n",
    "\n",
    "- remove duplicates and leaked videos\n",
    "- map `trainID` (0-721) to 0-399 (k400), 0-599 (k600), 0-699 (k700)\n",
    "\n",
    "The merged dataset coined Kinetics-722 (K-722) integrates all valid training samples from Kinetics-400 (K-400), Kinetics-600 (K-600) and Kinetics-700 (K-700).\n",
    "Notably, for a fair and legal comparison, we removed leaked videos in all validation sets and duplicated videos in all training sets based on `youtube id` of the video.\n",
    "Accordingly, the cleaned K-722 contains 0.63M training videos, covering 722 human action classes. We also provide [our data list]( https://huggingface.co/BAAI/EVA/blob/main/eva%20video%20data%20list.zip)."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from decord import VideoReader\n",
    "from decord import cpu\n",
    "\n",
    "\n",
    "DATA_ROOT = '../data'\n",
    "\n",
    "PATH_k400 = {'train': 'k400/train', 'val': 'k400/val'}\n",
    "PATH_k600 = {'train': 'k600/train', 'val': 'k600/val'}\n",
    "PATH_k700 = {'train': 'k700/train', 'val': 'k700/val'}\n",
    "\n",
    "kinetics_class_list = {\n",
    "    'k400': {k: sorted(os.listdir(os.path.join(DATA_ROOT, PATH_k400[k]))) for k in ['train', 'val']},\n",
    "    'k600': {k: sorted(os.listdir(os.path.join(DATA_ROOT, PATH_k600[k]))) for k in ['train', 'val']},\n",
    "    'k700': {k: sorted(os.listdir(os.path.join(DATA_ROOT, PATH_k700[k]))) for k in ['train', 'val']},\n",
    "}\n",
    "\n",
    "total_classes = sorted(\n",
    "    set(kinetics_class_list['k400']['val']) | set(kinetics_class_list['k600']['val']) | set(\n",
    "        kinetics_class_list['k700']['val']))\n",
    "\n",
    "K722_CLASS_MAPPING = {val: i for i, val in enumerate(total_classes)}\n",
    "\n",
    "K400_CLASS_MAPPING = {val: i for i, val in enumerate(sorted(set(kinetics_class_list['k400']['val'])))}\n",
    "\n",
    "K600_CLASS_MAPPING = {val: i for i, val in enumerate(sorted(set(kinetics_class_list['k600']['val'])))}\n",
    "\n",
    "K700_CLASS_MAPPING = {val: i for i, val in enumerate(sorted(set(kinetics_class_list['k700']['val'])))}\n",
    "\n",
    "# map 722 classes to 400/600/700 classses\n",
    "K722_TO_K400_MAPPING = []\n",
    "K722_TO_K600_MAPPING = []\n",
    "K722_TO_K700_MAPPING = []\n",
    "for label, trainID in K722_CLASS_MAPPING.items():\n",
    "    if label in K400_CLASS_MAPPING:\n",
    "        K722_TO_K400_MAPPING.append(trainID)\n",
    "    if label in K600_CLASS_MAPPING:\n",
    "        K722_TO_K600_MAPPING.append(trainID)\n",
    "    if label in K700_CLASS_MAPPING:\n",
    "        K722_TO_K700_MAPPING.append(trainID)\n",
    "np.save(os.path.join(DATA_ROOT, 'k722_to_k400_mapping.npy'), np.array(K722_TO_K400_MAPPING))\n",
    "np.save(os.path.join(DATA_ROOT,'k722_to_k600_mapping.npy'), np.array(K722_TO_K600_MAPPING))\n",
    "np.save(os.path.join(DATA_ROOT,'k722_to_k700_mapping.npy'), np.array(K722_TO_K700_MAPPING))\n",
    "\n",
    "def name_to_trainid(name):\n",
    "    return K722_CLASS_MAPPING[name]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "kinetics_file_dict = {'k400': {split: [] for split in ['train', 'val']},\n",
    "                'k600': {split: [] for split in ['train', 'val']},\n",
    "                 'k700': {split: [] for split in ['train', 'val']},\n",
    "                 'k722': {split: [] for split in ['train', 'val']},\n",
    "                      }\n",
    "\n",
    "for kinetics_name, class_sets in kinetics_class_list.items():  # k400, {'train': {CLASSES}, 'val': {CLASSES}}\n",
    "    assert len(set(kinetics_class_list[kinetics_name]['train']) | set(kinetics_class_list[kinetics_name]['val'])) == len(kinetics_class_list[kinetics_name]['val'])\n",
    "    for split in ['train', 'val']:\n",
    "        for class_name in class_sets[split]:\n",
    "\n",
    "            for video_id in os.listdir(\n",
    "                    os.path.join(DATA_ROOT, kinetics_name, split, class_name)):\n",
    "\n",
    "                if len(video_id) < 2 or video_id.endswith('tar.gz') or video_id.startswith('.'):  # special cases in local dataset\n",
    "                    continue\n",
    "\n",
    "                kinetics_file_dict[kinetics_name][split].append({'trainID': name_to_trainid(class_name), 'label': class_name,\n",
    "                                                            'video_id': video_id, 'youtube_id': video_id[:11]})\n",
    "\n",
    "                kinetics_file_dict['k722'][split].append({'trainID': name_to_trainid(class_name), 'label': class_name,\n",
    "                                                     'video_id': video_id, 'youtube_id': video_id[:11],\n",
    "                                                     'set': kinetics_name})\n",
    "\n",
    "            print(kinetics_name, split, class_name, name_to_trainid(class_name))\n",
    "\n",
    "\n",
    "k400_train_raw = pd.DataFrame(kinetics_file_dict['k400']['train'],\n",
    "                              columns=['trainID', 'label', 'video_id', 'youtube_id'])\n",
    "k400_val_raw = pd.DataFrame(kinetics_file_dict['k400']['val'],\n",
    "                            columns=['trainID', 'label', 'video_id', 'youtube_id'])\n",
    "k600_train_raw = pd.DataFrame(kinetics_file_dict['k600']['train'],\n",
    "                              columns=['trainID', 'label', 'video_id', 'youtube_id'])\n",
    "k600_val_raw = pd.DataFrame(kinetics_file_dict['k600']['val'],\n",
    "                            columns=['trainID', 'label', 'video_id', 'youtube_id'])\n",
    "k700_train_raw = pd.DataFrame(kinetics_file_dict['k700']['train'],\n",
    "                              columns=['trainID', 'label', 'video_id', 'youtube_id'])\n",
    "k700_val_raw = pd.DataFrame(kinetics_file_dict['k700']['val'],\n",
    "                            columns=['trainID', 'label', 'video_id', 'youtube_id'])\n",
    "k722_train_raw = pd.DataFrame(kinetics_file_dict['k722']['train'],\n",
    "                              columns=['trainID', 'label', 'video_id', 'set', 'youtube_id'])\n",
    "k722_val_raw = pd.DataFrame(kinetics_file_dict['k722']['val'],\n",
    "                            columns=['trainID', 'label', 'video_id', 'set', 'youtube_id'])\n",
    "\n",
    "# Kinetics400 data list precessing\n",
    "print('-' * 66)\n",
    "print(f'k400: train: {len(k400_train_raw)}, val: {len(k400_val_raw)}')\n",
    "k400_train_raw = k400_train_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('after drop repeated video in k400 train set:', len(k400_train_raw))\n",
    "print('after drop leaked video in k400 val set:', len(k400_train_raw))\n",
    "k400_val_raw = k400_val_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('after drop repeated video in k400 val set:', len(k400_val_raw))\n",
    "\n",
    "with open(f'{DATA_ROOT}/k400_train_raw.txt', 'w') as f:\n",
    "    for index, row in k400_train_raw.iterrows():\n",
    "        item = f\"{os.path.join('k400/train', row['label'], row['video_id'])} {K722_TO_K400_MAPPING.index(name_to_trainid(row['label']))}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "with open(f'{DATA_ROOT}/k400_val_raw.txt', 'w') as f:\n",
    "    for index, row in k400_val_raw.iterrows():\n",
    "        item = f\"{os.path.join('k400/val', row['label'], row['video_id'])} {K722_TO_K400_MAPPING.index(name_to_trainid(row['label']))}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "\n",
    "# Kinetics600 data list precessing\n",
    "print('-' * 66)\n",
    "print(f'k600: train: {len(k600_train_raw)}, val: {len(k600_val_raw)}')\n",
    "k600_train_raw = k600_train_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('after drop repeated video in k600 train set:', len(k600_train_raw))\n",
    "k600_train_raw = k600_train_raw[\n",
    "    ~k600_train_raw['youtube_id'].isin(k600_val_raw['youtube_id'])]\n",
    "print('after drop leaked video in k600 val set:', len(k600_train_raw))\n",
    "k600_val_raw = k600_val_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('after drop repeated video in k600 val set:', len(k600_val_raw))\n",
    "\n",
    "with open(f'{DATA_ROOT}/k600_train_raw.txt', 'w') as f:\n",
    "    for index, row in k600_train_raw.iterrows():\n",
    "        item = f\"{os.path.join('k600/train', row['label'], row['video_id'])} {K722_TO_K600_MAPPING.index(name_to_trainid(row['label']))}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "with open(f'{DATA_ROOT}/k600_val_raw.txt', 'w') as f:\n",
    "    for index, row in k600_val_raw.iterrows():\n",
    "        item = f\"{os.path.join('k600/val', row['label'], row['video_id'])} {K722_TO_K600_MAPPING.index(name_to_trainid(row['label']))}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "\n",
    "# Kinetics700 data list precessing\n",
    "print('-' * 66)\n",
    "print(f'k700: train: {len(k700_train_raw)}, val: {len(k700_val_raw)}')\n",
    "k700_train_raw = k700_train_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('after drop repeated video in k700 train set:', len(k700_train_raw))\n",
    "k700_train_raw = k700_train_raw[\n",
    "    ~k700_train_raw['youtube_id'].isin(k700_val_raw['youtube_id'])]\n",
    "print('after drop leaked video in k700 val set:', len(k700_train_raw))\n",
    "k700_val_raw = k700_val_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('after drop repeated video in k700 val set:', len(k700_val_raw))\n",
    "\n",
    "with open(f'{DATA_ROOT}/k700_train_raw.txt', 'w') as f:\n",
    "    for index, row in k700_train_raw.iterrows():\n",
    "        item = f\"{os.path.join('k700/train', row['label'], row['video_id'])} {K722_TO_K700_MAPPING.index(name_to_trainid(row['label']))}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "with open(f'{DATA_ROOT}/k700_val_raw.txt', 'w') as f:\n",
    "    for index, row in k700_val_raw.iterrows():\n",
    "        item = f\"{os.path.join('k700/val', row['label'], row['video_id'])} {K722_TO_K700_MAPPING.index(name_to_trainid(row['label']))}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "\n",
    "\n",
    "# Kinetics722 data list precessing\n",
    "print('-' * 66)\n",
    "print('k722 train length:', len(kinetics_file_dict['k722']['train']))\n",
    "print(f\"k722 val length: {len(kinetics_file_dict['k722']['val'])}\")\n",
    "print('-' * 66)\n",
    "\n",
    "print('drop repeated video in k722 val set')\n",
    "k722_val_drop = k722_val_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('before drop duplicates (val): ', len(k722_val_raw))\n",
    "print('after drop duplicates (val):', len(k722_val_drop))\n",
    "print('-' * 66)\n",
    "\n",
    "print('drop repeated video in k722 train set')\n",
    "k722_train_drop = k722_train_raw.drop_duplicates(subset=['youtube_id'])\n",
    "print('before drop duplicates: ', len(k722_train_raw))\n",
    "print('after drop duplicates:', len(k722_train_drop))\n",
    "print('-' * 66)\n",
    "\n",
    "print('drop leaked video in k400/600/700 val set')\n",
    "kinectics_val = {kinectics: pd.DataFrame(kinetics_file_dict[kinectics]['val'], columns=['label', 'video_id', 'youtube_id'])\n",
    "                 for kinectics in ['k400', 'k600', 'k700']}\n",
    "kinectics_train = {\n",
    "    kinectics: pd.DataFrame(kinetics_file_dict[kinectics]['train'], columns=['label', 'video_id', 'youtube_id']) for\n",
    "    kinectics in ['k400', 'k600', 'k700']}\n",
    "\n",
    "k722_train_drop_leak = k722_train_drop.copy(deep=True)\n",
    "for kinetics_name, val_csv in kinectics_val.items():\n",
    "    k722_train_drop_leak = k722_train_drop_leak[\n",
    "        ~ k722_train_drop_leak['youtube_id'].isin(kinectics_val[kinetics_name]['youtube_id'])]\n",
    "    print(f'after drop {kinetics_name} validation:', len(k722_train_drop_leak))\n",
    "\n",
    "with open(f'{DATA_ROOT}/k722_train_raw.txt', 'w') as f:\n",
    "    for index, row in k722_train_drop_leak.iterrows():\n",
    "        item = f\"{os.path.join(row['set'], 'train', row['label'], row['video_id'])} {name_to_trainid(row['label'])}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "with open(f'{DATA_ROOT}/k722_val_raw.txt', 'w') as f:\n",
    "    for index, row in k722_val_drop.iterrows():\n",
    "        item = f\"{os.path.join(row['set'], 'val', row['label'], row['video_id'])} {name_to_trainid(row['label'])}\\n\"\n",
    "        f.write(item)\n",
    "f.close()\n",
    "\n",
    "print('-' * 66)\n",
    "print('right now our k722')\n",
    "print('class num:', len(k722_train_drop_leak['label'].unique()))\n",
    "print('k722 train video:', len(k722_train_drop_leak['label']))\n",
    "print('k722 val video:', len(k722_val_drop['label']))\n",
    "\n",
    "print('\\nright now our k700')\n",
    "print('class num:', len(k700_train_raw['label'].unique()))\n",
    "print('k722 train video:', len(k700_train_raw['label']))\n",
    "print('k722 val video:', len(k700_val_raw['label']))\n",
    "\n",
    "print('\\nright now our k600')\n",
    "print('class num:', len(k600_train_raw['label'].unique()))\n",
    "print('k722 train video:', len(k600_train_raw['label']))\n",
    "print('k722 val video:', len(k600_val_raw['label']))\n",
    "\n",
    "print('\\nright now our k400')\n",
    "print('class num:', len(k400_train_raw['label'].unique()))\n",
    "print('k722 train video:', len(k400_train_raw['label']))\n",
    "print('k722 val video:', len(k400_val_raw['label']))\n",
    "print('-' * 66)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Confirm that videos are correct"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "KINECTICS = ['k400', 'k600', 'k700', 'k722']\n",
    "SETS = ['train', 'val']\n",
    "\n",
    "print('Confirm that videos are correct.')\n",
    "for kinetics in KINECTICS:\n",
    "    for set in SETS:\n",
    "        print(f'{kinetics}\\t{set}:')\n",
    "        total = 0\n",
    "        video_infos = []\n",
    "        ann_file = os.path.join(DATA_ROOT, f'{kinetics}_{set}_raw.txt')\n",
    "        print(f'processing {ann_file}')\n",
    "        with open(ann_file, 'r') as f:\n",
    "            for line in f:\n",
    "                line_split = line.strip().split()\n",
    "                filename = ' '.join(line_split[:-1])\n",
    "                label = int(line_split[-1])\n",
    "                video_infos.append(dict(filename=filename, label=label))\n",
    "                total += 1\n",
    "        f.close()\n",
    "\n",
    "        correct = []\n",
    "        error = []\n",
    "        for video in video_infos:\n",
    "            file_name = video['filename']\n",
    "            label = video['label']\n",
    "            try:\n",
    "                vr = VideoReader(os.path.join(DATA_ROOT, file_name), ctx=cpu(0))\n",
    "                correct.append(f'{file_name} {label}')\n",
    "            except:\n",
    "                print(f'loading filed {file_name}')\n",
    "                error.append(file_name)\n",
    "\n",
    "        print(f'total: {total}\\t correct:{len(correct)}\\t error: {len(error)}')\n",
    "\n",
    "        with open(os.path.join(DATA_ROOT, f'{ann_file.replace(\"_raw\", \"\")}'), 'w') as f:\n",
    "            for c in correct:\n",
    "                f.write(f'{c}\\n')\n",
    "        f.close()\n",
    "\n",
    "        with open(os.path.join(DATA_ROOT, f'{ann_file.replace(\"_raw\", \"_error\")}'), 'w') as f:\n",
    "            for e in error:\n",
    "                f.write(f'{e}\\n')\n",
    "        f.close()\n"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
